package com.fwmagic.spark.core.transformations

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * groupByKey会将RDD[key,value] 按照相同的key进行分组，
  * 形成RDD[key,Iterable[value]]的形式，
  * 有点类似于sql中的groupby，例如类似于mysql中的group_concat
  */
object GroupByKeyDemo {
    def main(args: Array[String]): Unit = {
        val conf: SparkConf = new SparkConf()
                .setAppName(this.getClass.getSimpleName)
                .setMaster("local[*]")

        val sc: SparkContext = new SparkContext(conf)

        val rdd: RDD[(String, Int)] = sc.parallelize(List(("xiaoming",75),("xiaoming",90),("lihua",95),("lihua",100),("xiaofeng",1),("xiaofeng",85)))

        val rdd2: RDD[(String, Iterable[Int])] = rdd.groupByKey()

        val sumed: RDD[(String, Int)] = rdd2.mapValues(_.sum)
        //val rdd2: RDD[(String, Iterable[(String, Int)])] = rdd.groupBy(_._1)

        rdd2.collect().foreach(println)

        sumed.collect().foreach(println)
        sc.stop()

    }

}
